evalscope 1.0.0__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/api/benchmark/__init__.py

@@ -1,3 +1,3 @@
-from .adapters import DefaultDataAdapter, MultiChoiceAdapter, Text2ImageAdapter
+from .adapters import DefaultDataAdapter, ImageEditAdapter, MultiChoiceAdapter, Text2ImageAdapter, VisionLanguageAdapter
 from .benchmark import DataAdapter
 from .meta import BenchmarkMeta

evalscope/api/benchmark/adapters/__init__.py

@@ -1,3 +1,5 @@
 from .default_data_adapter import DefaultDataAdapter
+from .image_edit_adapter import ImageEditAdapter
 from .multi_choice_adapter import MultiChoiceAdapter
 from .text2image_adapter import Text2ImageAdapter
+from .vision_language_adapter import VisionLanguageAdapter

evalscope/api/benchmark/adapters/default_data_adapter.py

@@ -241,6 +241,7 @@ class DefaultDataAdapter(DataAdapter):
             filter_func=self.sample_filter,
             limit=self.limit if not self.reformat_subset else None,  # Limit number of samples if specified
             repeats=self.repeats,  # Number of repetitions for each sample
+            shuffle=self.shuffle,  # Shuffle dataset if enabled
             shuffle_choices=self.shuffle_choices,  # Shuffle choices if requested
             data_source=self.dataset_hub,  # Data source configuration
         )

evalscope/api/benchmark/adapters/image_edit_adapter.py

@@ -0,0 +1,82 @@
+import os
+from typing import Optional
+
+from evalscope.constants import EvalType, FileConstants
+from evalscope.utils import get_logger
+from evalscope.utils.function_utils import thread_safe
+from evalscope.utils.io_utils import jsonl_to_list
+from .text2image_adapter import Text2ImageAdapter
+
+logger = get_logger()
+
+
+class ImageEditAdapter(Text2ImageAdapter):
+    """
+    Support two methods:
+    1. Inference using modelscope pipeline
+    2. Load local inference jsonl file with key to corresponding prompt
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.local_file = self.extra_params.get('local_file', None)
+        self.id_key = self.extra_params.get('id_key', FileConstants.ID)
+        self.image_key = self.extra_params.get('image_key', FileConstants.IMAGE_PATH)
+        self.local_data = self.load_local_file()
+
+    def load_local_file(self) -> Optional[dict]:
+        if not self.local_file:
+            return None
+
+        # Load file and check
+        data_list = jsonl_to_list(self.local_file)
+        data_dict = {}
+        for record in data_list:
+            if self.image_key not in record:
+                raise ValueError(f"Image key '{self.image_key}' not found in record: {record}, file {self.local_file}")
+            if self.id_key not in record:
+                raise ValueError(f"ID key '{self.id_key}' not found in record: {record}, file {self.local_file}")
+
+            image_path = record[self.image_key]
+            if not os.path.isabs(image_path):
+                image_path = os.path.join(os.path.dirname(self.local_file), image_path)
+            if not os.path.exists(image_path):
+                raise FileNotFoundError(f"Image file '{image_path}' not found.")
+
+            data_dict[record[self.id_key]] = record
+        return data_dict
+
+    def get_image_path_from_id(self, image_id) -> Optional[str]:
+        if not self.local_file:
+            return None
+
+        record = self.local_data.get(image_id)
+        if not record:
+            return None
+
+        return record[self.image_key]
+
+    def _post_process_samples(self):
+        super()._post_process_samples()
+
+        # Add local image path if exists
+        for subset in self.test_dataset.keys():
+            for sample in self.test_dataset[subset]:
+                local_image_path = self.get_image_path_from_id(sample.metadata.get(FileConstants.ID))
+                if local_image_path:
+                    sample.metadata[FileConstants.IMAGE_PATH] = local_image_path
+
+    def sample_filter(self, sample) -> bool:
+        """
+        Filter samples based on metadata availability.
+        If local file is not available, all samples are considered valid.
+        Otherwise, only samples with valid metadata and image path are kept.
+        """
+        if not self.local_data:
+            return True
+        else:
+            sample_id = sample.metadata.get(FileConstants.ID)
+            if (not sample_id) or (not self.get_image_path_from_id(sample_id)):
+                return False
+            return True

evalscope/api/benchmark/adapters/text2image_adapter.py

@@ -8,7 +8,7 @@ from evalscope.api.messages.content import ContentImage
 from evalscope.api.metric import Score
 from evalscope.api.model import ChatCompletionChoice, Model, ModelOutput
 from evalscope.api.registry import get_metric
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, FileConstants
 from evalscope.utils import get_logger
 from evalscope.utils.function_utils import thread_safe
 from .default_data_adapter import DefaultDataAdapter
@@ -27,11 +27,12 @@ class Text2ImageAdapter(DefaultDataAdapter):
         return Sample(
             input=[ChatMessageUser(content=record['prompt'])],
             metadata={
-                'id': record['id'],
                 'prompt': record['prompt'],
                 'category': record.get('category', ''),
                 'tags': record.get('tags', []),
-                'image_path': record.get('image_path', ''),  # Optional field for existing image path
+                FileConstants.ID: record[FileConstants.ID],
+                FileConstants.IMAGE_PATH: record.get(FileConstants.IMAGE_PATH,
+                                                     ''),  # Optional field for existing image path
             }
         )
 
@@ -83,7 +84,7 @@ class Text2ImageAdapter(DefaultDataAdapter):
                 completed=True,
             )
         else:
-            image_id = f"{sample.metadata.get('id',sample.id)}_{sample.group_id}"
+            image_id = f'{sample.metadata.get(FileConstants.ID, sample.id)}_{sample.group_id}'
             output_path = os.path.join(output_dir, 'images', f'{image_id}.png')
             if not os.path.exists(os.path.dirname(output_path)):
                 os.makedirs(os.path.dirname(output_path))
@@ -96,7 +97,7 @@ class Text2ImageAdapter(DefaultDataAdapter):
             with open(output_path, 'wb') as f:
                 f.write(base64.b64decode(image_base64))
 
-            sample.metadata['image_path'] = output_path
+            sample.metadata[FileConstants.IMAGE_PATH] = output_path
             return TaskState(
                 model=model.name,
                 sample=sample,
@@ -111,7 +112,7 @@ class Text2ImageAdapter(DefaultDataAdapter):
         self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
     ) -> Score:
         # Get prediction and prompt from task state
-        image_path = task_state.metadata.get('image_path', original_prediction)
+        image_path = task_state.metadata.get(FileConstants.IMAGE_PATH, original_prediction)
         prompt = task_state.input[0].content
         meta = task_state.metadata
 

evalscope/api/benchmark/adapters/vision_language_adapter.py

@@ -0,0 +1,6 @@
+from .default_data_adapter import DefaultDataAdapter
+
+
+class VisionLanguageAdapter(DefaultDataAdapter):
+    """Adapter for vision-language benchmarks. e.g., image captioning, visual question answering, etc."""
+    pass

evalscope/api/benchmark/benchmark.py

@@ -170,6 +170,13 @@ class DataAdapter(LLMJudgeMixin, ABC):
         """
         return self._benchmark_meta.default_subset
 
+    @default_subset.setter
+    def default_subset(self, value: str):
+        """
+        Set the default subset of the benchmark.
+        """
+        self._benchmark_meta.default_subset = value
+
     @property
     def few_shot_num(self) -> int:
         """
@@ -299,6 +306,34 @@ class DataAdapter(LLMJudgeMixin, ABC):
         """
         return self._task_config.seed
 
+    @property
+    def shuffle(self) -> bool:
+        """
+        Return whether to shuffle the dataset before evaluation.
+        """
+        return self._benchmark_meta.shuffle
+
+    @shuffle.setter
+    def shuffle(self, value: bool):
+        """
+        Set whether to shuffle the dataset before evaluation.
+        """
+        self._benchmark_meta.shuffle = value
+
+    @property
+    def shuffle_choices(self) -> bool:
+        """
+        Return whether to shuffle the choices in multiple-choice datasets.
+        """
+        return self._benchmark_meta.shuffle_choices
+
+    @shuffle_choices.setter
+    def shuffle_choices(self, value: bool):
+        """
+        Set whether to shuffle the choices in multiple-choice datasets.
+        """
+        self._benchmark_meta.shuffle_choices = value
+
     @contextlib.contextmanager
     def _temporary_attribute(self, attr_name: str, new_value):
         """

evalscope/api/benchmark/meta.py

@@ -73,6 +73,12 @@ class BenchmarkMeta:
     aggregation: str = 'mean'
     """ Aggregation function for the metrics. Default is 'mean'. Can be 'mean', 'pass@<k>' or a custom function name."""
 
+    shuffle: bool = False
+    """Whether to shuffle the dataset before evaluation."""
+
+    shuffle_choices: bool = False
+    """Whether to shuffle the choices in multiple-choice datasets."""
+
     extra_params: Dict = field(default_factory=dict)
     """ Additional parameters for the benchmark."""
 

evalscope/api/dataset/dataset.py

@@ -5,9 +5,8 @@ from dataclasses import dataclass, field
 from pydantic import BaseModel, Field
 from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Union
 
-from evalscope.api.messages import ChatMessage, messages_pretty_str
+from evalscope.api.messages import ChatMessage, messages_to_markdown
 from evalscope.api.tool import ToolInfo
-from evalscope.utils.multi_choices import answer_character, answer_index
 
 
 class Sample(BaseModel):
@@ -31,9 +30,6 @@ class Sample(BaseModel):
     tools: Optional[List[ToolInfo]] = None
     """List of tools available to the model during inference (optional)."""
 
-    category: Optional[str] = None
-    """Category of the sample (optional)."""
-
     subset_key: Optional[str] = None
     """Key for the subset this sample belongs to, used for generating subsets (optional)."""
 
@@ -54,7 +50,7 @@ class Sample(BaseModel):
         if isinstance(self.input, str):
             input_text = self.input
         else:
-            input_text = messages_pretty_str(self.input)
+            input_text = messages_to_markdown(self.input, max_length=50)
         return f'Sample ID: {self.id}\nInput: {input_text}\nTarget: {self.target}'
 
 
@@ -230,6 +226,8 @@ class MemoryDataset(Dataset):
         self._shuffled = True
 
     def shuffle_choices(self, seed: Optional[int] = None) -> None:
+        from evalscope.utils.multi_choices import answer_character
+
         rand = random.Random(seed)
         for sample in self.samples:
             if not sample.choices:
@@ -249,6 +247,8 @@ class MemoryDataset(Dataset):
             sample.target = self._remap_target(sample.target, position_map=position_map)
 
     def _remap_target(self, target: Union[str, List[str]], position_map: Dict[int, str]) -> Union[str, List[str]]:
+        from evalscope.utils.multi_choices import answer_index
+
         if isinstance(target, list):
             return [position_map[answer_index(t)] for t in target]
         else:

evalscope/api/dataset/loader.py

@@ -126,7 +126,8 @@ class RemoteDataLoader(DataLoader):
                 self.limit = int(len(dataset) * self.limit)
             elif isinstance(self.limit, int) and self.limit < 0:
                 raise ValueError('Limit must be a non-negative integer or a float between 0 and 1.')
-            dataset = dataset.select(range(self.limit))
+            if len(dataset) > self.limit:
+                dataset = dataset.select(range(self.limit))
 
         # convert to list
         dataset = dataset.to_list()

evalscope/api/evaluator/cache.py

@@ -299,6 +299,15 @@ class ModelResult(BaseModel):
             completed=True,  # Mark as completed since it was cached
         )
 
+    def pretty_print(self) -> str:
+        """
+        Generate a pretty-printed string representation of the model result.
+
+        Returns:
+            A string representation of the model result
+        """
+        return self.model_dump_json(indent=2)
+
 
 class ReviewResult(BaseModel):
     """
@@ -340,7 +349,7 @@ class ReviewResult(BaseModel):
 
         return cls(
             index=state.sample_id,
-            input=state.input_text,
+            input=state.input_markdown,
             target=state.target,
             sample_score=sample_score,
         )
@@ -353,3 +362,17 @@ class ReviewResult(BaseModel):
             The sample score object
         """
         return self.sample_score
+
+    def pretty_print(self) -> str:
+        """
+        Generate a pretty-printed string representation of the review result.
+
+        Returns:
+            A string representation of the review result
+        """
+        output = [
+            f'Review Result for Sample {self.index}:',
+            f'Target: {self.target}',
+            f'Score: {self.sample_score.model_dump_json(indent=2)}',
+        ]
+        return '\n'.join(output)

evalscope/api/evaluator/state.py

@@ -3,7 +3,7 @@ from random import Random
 from typing import Any, Dict, List, Optional, Sequence, Union, overload
 
 from evalscope.api.dataset import Sample
-from evalscope.api.messages import ChatMessage, ChatMessageUser, messages_pretty_str
+from evalscope.api.messages import ChatMessage, ChatMessageUser, messages_pretty_str, messages_to_markdown
 from evalscope.api.model import ModelOutput
 
 
@@ -188,6 +188,17 @@ class TaskState:
         else:
             return messages_pretty_str(self._input)
 
+    @property
+    def input_markdown(self) -> str:
+        """Get the input text as markdown.
+
+        For multi-modal content, images will be represented in markdown format.
+        """
+        if isinstance(self._input, str):
+            return self._input
+        else:
+            return messages_to_markdown(self._input)
+
     @property
     def choices(self) -> Choices:
         """Choices for the sample, if applicable."""

evalscope/api/messages/__init__.py

@@ -6,6 +6,7 @@ from .chat_message import (
     ChatMessageUser,
     dict_to_chat_message,
     messages_pretty_str,
+    messages_to_markdown,
 )
 from .content import Content, ContentAudio, ContentData, ContentImage, ContentReasoning, ContentText, ContentVideo
 from .utils import parse_content_with_reasoning

evalscope/api/messages/chat_message.py

@@ -3,7 +3,7 @@ from pydantic import BaseModel, Field, JsonValue, model_validator
 from typing import Any, Dict, List, Literal, Optional, Type, Union
 
 from evalscope.api.tool import ToolCall, ToolCallError
-from .content import Content, ContentReasoning, ContentText
+from .content import Content, ContentImage, ContentReasoning, ContentText
 from .utils import parse_content_with_reasoning
 
 
@@ -184,7 +184,7 @@ def dict_to_chat_message(data: Dict[str, Any]) -> ChatMessage:
 
 
 def messages_pretty_str(messages: List[ChatMessage]) -> str:
-    """Pretty print a list of chat messages."""
+    """Pretty print a list of chat messages. Without images or other multi-modal contents."""
     output = []
     for message in messages:
         role = message.role.capitalize()
@@ -196,3 +196,48 @@ def messages_pretty_str(messages: List[ChatMessage]) -> str:
                 content += f'\nFunction: {message.function}'
         output.append(f'**{role}**: {content}')
     return '\n\n'.join(output)
+
+
+def messages_to_markdown(messages: List[ChatMessage], max_length: Optional[int] = None) -> str:
+    """Convert a list of chat messages to markdown format.
+
+    Args:
+        messages (List[ChatMessage]): The list of chat messages to convert.
+        max_length (Optional[int]): If provided, truncates the base64 string of images to this length.
+    """
+    output = []
+    for message in messages:
+        role = message.role.capitalize()
+
+        # Start with role header
+        content_parts = [f'**{role}**: ']
+
+        # Handle content based on type
+        if isinstance(message.content, str):
+            content_parts.append(message.content)
+        else:
+            for content_item in message.content:
+                if isinstance(content_item, ContentText):
+                    content_parts.append(content_item.text)
+                elif isinstance(content_item, ContentImage):
+                    # Use markdown image syntax
+                    image_base64 = content_item.image
+                    if max_length and len(image_base64) > max_length:
+                        image_base64 = image_base64[:max_length]
+                    content_parts.append(f'![image]({image_base64})')
+                elif isinstance(content_item, ContentReasoning):
+                    content_parts.append(f'**Reasoning:** {content_item.reasoning}')
+
+        # Add tool-specific information
+        if isinstance(message, ChatMessageTool):
+            if message.error:
+                content_parts.append(f'**Error:** {message.error.message}')
+            if message.function:
+                content_parts.append(f'**Function:** {message.function}')
+        elif isinstance(message, ChatMessageAssistant) and message.tool_calls:
+            for tool_call in message.tool_calls:
+                content_parts.append(f'**Tool Call:** {tool_call.function}')
+
+        output.append('\n'.join(content_parts))
+
+    return '\n\n'.join(output)

evalscope/api/metric/scorer.py

@@ -35,20 +35,28 @@ class Score(BaseModel):
         """Main score value."""
         if self.main_score_name and self.main_score_name in self.value:
             return self.value[self.main_score_name]
-        return next(iter(self.value.values()), None)
+        elif self.value:
+            # If main_score_name is not set or not found, use the first value and update main_score_name
+            first_key = next(iter(self.value))
+            self.main_score_name = first_key
+            return self.value[first_key]
+        return None
 
     @main_value.setter
     def main_value(self, value: Union[int, float, bool]):
         """Set the main score value."""
         if self.main_score_name:
+            # If main_score_name is already set, use it
             self.value[self.main_score_name] = value
+        elif self.value:
+            # If no main_score_name but value dict exists, use the first key
+            first_key = next(iter(self.value))
+            self.main_score_name = first_key
+            self.value[first_key] = value
         else:
-            # If no main score name is set, just update the first value
-            if self.value:
-                first_key = next(iter(self.value))
-                self.value[first_key] = value
-            else:
-                self.value['default'] = value
+            # If neither main_score_name nor value dict exists, initialize both
+            self.main_score_name = 'default'
+            self.value[self.main_score_name] = value
 
 
 class SampleScore(BaseModel):

evalscope/api/mixin/__init__.py

@@ -1,2 +1 @@
-from .dataset_mixin import DatasetLoaderMixin
 from .llm_judge_mixin import LLMJudgeMixin

evalscope/api/model/generate_config.py

@@ -25,9 +25,7 @@ class ResponseSchema(BaseModel):
 
 class GenerateConfig(BaseModel):
     """Model generation options."""
-
-    max_retries: Optional[int] = Field(default=None)
-    """Maximum number of times to retry request (defaults to unlimited)."""
+    model_config = {'extra': 'allow'}
 
     timeout: Optional[int] = Field(default=None)
     """Request timeout (in seconds)."""

evalscope/api/model/model.py

@@ -318,7 +318,7 @@ def get_model_with_task_config(task_config: 'TaskConfig') -> Model:
 
 @thread_safe
 def get_model(
-    model: str,
+    model: Union[str, Model, ModelAPI],
     eval_type: str,
     base_url: Optional[str] = None,
     api_key: Optional[str] = None,
@@ -346,6 +346,9 @@ def get_model(
     if isinstance(model, Model):
         return model
 
+    if isinstance(model, ModelAPI):
+        return Model(model, config, model_args)
+
     # see if we can return a memoized model instance
     # (exclude mockllm since custom_outputs is an infinite generator)
     model_cache_key: str = ''

evalscope/app/app.py

@@ -6,6 +6,7 @@ import argparse
 from evalscope.utils.logger import configure_logging
 from .arguments import add_argument
 from .ui import create_app_ui
+from .utils.env_utils import setup_env
 
 
 def create_app(args: argparse.Namespace):
@@ -17,6 +18,8 @@ def create_app(args: argparse.Namespace):
     """
     configure_logging(debug=args.debug)
 
+    setup_env(args)
+
     demo = create_app_ui(args)
 
     demo.launch(

evalscope/app/ui/single_model.py

@@ -198,9 +198,9 @@ def create_single_model_tab(sidebar: 'SidebarComponents', lang: str):
 
         # Process the data for display
         input_md = row['Input'] + '\n\n' + process_model_prediction(row['Metadata'])
-        generated_md = process_model_prediction(row['Generated'])
-        gold_md = process_model_prediction(row['Gold'])
-        pred_md = convert_markdown_image(process_model_prediction(row['Pred']))
+        generated_md = convert_markdown_image(row['Generated'])
+        gold_md = convert_markdown_image(row['Gold'])
+        pred_md = process_model_prediction(row['Pred'])
         score_md = process_json_content(row['Score'])
         nscore_val = float(row['NScore']) if not pd.isna(row['NScore']) else 0.0
 

evalscope/app/utils/data_utils.py

@@ -2,7 +2,6 @@
 Data loading and processing utilities for the Evalscope dashboard.
 """
 import glob
-import numpy as np
 import os
 import pandas as pd
 from typing import Any, Dict, List, Union
@@ -160,17 +159,18 @@ def get_model_prediction(work_dir: str, model_name: str, dataset_name: str, subs
             if f'{sample_dataset_name}/{sample_subset_name}' != subset_name:
                 continue
 
-        prediction = sample_score.score.prediction
-        target = review_result.target
-        extracted_prediction = sample_score.score.extracted_prediction
         score = sample_score.score
+        metadata = sample_score.sample_metadata
+        prediction = score.prediction
+        target = review_result.target
+        extracted_prediction = score.extracted_prediction
         raw_d = {
             'Index': str(review_result.index),
             'Input': review_result.input.replace('\n', '\n\n'),  # for markdown
-            'Metadata': sample_score.sample_metadata,
-            'Generated': prediction if prediction != extracted_prediction else '*Same as Pred*',
+            'Metadata': metadata,
+            'Generated': prediction,
             'Gold': target,
-            'Pred': extracted_prediction,
+            'Pred': extracted_prediction if extracted_prediction != prediction else '*Same as Generated*',
             'Score': score.model_dump(exclude_none=True),
             'NScore': normalize_score(score.main_value)
         }

evalscope/app/utils/env_utils.py

@@ -0,0 +1,12 @@
+# flake8: noqa
+import os
+
+
+def setup_env(args):
+    compat_dsw_gradio(args)
+
+
+def compat_dsw_gradio(args) -> None:
+    if ('JUPYTER_NAME' in os.environ) and ('dsw-'
+                                           in os.environ['JUPYTER_NAME']) and ('GRADIO_ROOT_PATH' not in os.environ):
+        os.environ['GRADIO_ROOT_PATH'] = f"/{os.environ['JUPYTER_NAME']}/proxy/{args.server_port}"

evalscope/app/utils/text_utils.py

@@ -2,11 +2,9 @@
 Text processing utilities for the Evalscope dashboard.
 """
 import json
-import numpy as np
 import os
-import pandas as pd
 import re
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 from evalscope.utils.logger import get_logger
 from ..constants import LATEX_DELIMITERS
@@ -14,15 +12,19 @@ from ..constants import LATEX_DELIMITERS
 logger = get_logger()
 
 
-def convert_markdown_image(text):
-    if not os.path.isfile(text):
-        return text
-    # Convert the image path to a markdown image tag
-    if text.endswith('.png') or text.endswith('.jpg') or text.endswith('.jpeg'):
-        text = os.path.abspath(text)
-        image_tag = f'![image](gradio_api/file={text})'
-        logger.debug(f'Converting image path to markdown: {text} -> {image_tag}')
+def convert_markdown_image(text: str):
+    if text.startswith('data:image'):
+        # Convert base64 image data to a markdown image tag
+        image_tag = f'![image]({text})'
+        logger.debug(f'Converting base64 image data to markdown: {text[:30]}... -> {image_tag[:40]}...')
         return image_tag
+    elif os.path.isfile(text):
+        # Convert the image path to a markdown image tag
+        if text.endswith('.png') or text.endswith('.jpg') or text.endswith('.jpeg'):
+            text = os.path.abspath(text)
+            image_tag = f'![image](gradio_api/file={text})'
+            logger.debug(f'Converting image path to markdown: {text} -> {image_tag}')
+            return image_tag
     return text
 
 
@@ -85,7 +87,7 @@ def process_model_prediction_old(item: Any, max_length: int = 2048) -> str:
     return result
 
 
-def process_model_prediction(item: Any, max_length: int = 32000) -> str:
+def process_model_prediction(item: Any, max_length: Optional[int] = None) -> str:
     if isinstance(item, (dict, list)):
         result = json.dumps(item, ensure_ascii=False, indent=2)
         result = f'```json\n{result}\n```'

evalscope/arguments.py

@@ -2,7 +2,7 @@
 import argparse
 import json
 
-from evalscope.constants import EvalBackend, EvalType, JudgeStrategy, ModelTask, OutputType
+from evalscope.constants import EvalBackend, EvalType, JudgeStrategy, ModelTask
 
 
 class ParseStrArgsAction(argparse.Action):
@@ -60,8 +60,7 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--generation-config', type=str, action=ParseStrArgsAction, help='The generation config, should be a string.')  # noqa: E501
 
     # Evaluation-related arguments
-    parser.add_argument('--eval-type', type=str, help='The type for evaluating.',
-                        choices=[EvalType.CHECKPOINT, EvalType.CUSTOM, EvalType.SERVICE])
+    parser.add_argument('--eval-type', type=str, help='The type for evaluating.')
     parser.add_argument('--eval-backend', type=str, help='The evaluation backend to use.',
                         choices=[EvalBackend.NATIVE, EvalBackend.OPEN_COMPASS, EvalBackend.VLM_EVAL_KIT, EvalBackend.RAG_EVAL])  # noqa: E501
     parser.add_argument('--eval-config', type=str, required=False, help='The eval task config file path for evaluation backend.')  # noqa: E501
@@ -77,7 +76,6 @@ def add_argument(parser: argparse.ArgumentParser):
     # Debug and runtime mode arguments
     parser.add_argument('--ignore-errors', action='store_true', default=False, help='Ignore errors during evaluation.')
     parser.add_argument('--debug', action='store_true', default=False, help='Debug mode, will print information for debugging.')  # noqa: E501
-    parser.add_argument('--dry-run', action='store_true', default=False, help='Dry run in single processing mode.')
     parser.add_argument('--seed', type=int, default=42, help='Random seed for reproducibility.')
     parser.add_argument('--api-key', type=str, default='EMPTY', help='The API key for the remote API model.')
     parser.add_argument('--api-url', type=str, default=None, help='The API url for the remote API model.')